# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals

# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter

from selenium import webdriver
from selenium.common.exceptions import JavascriptException
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver import ActionChains



from scrapy.http import HtmlResponse

import time

import random

from scrapy.http import HtmlResponse

# 定义一组代理IP
# http://www.geetest.com/first_page 这是一个对手

PROXIES = [
    {'ip_port': '61.160.233.8', 'user_pass': ''},
    {'ip_port': '125.93.149.186', 'user_pass': ''},
    {'ip_port': '58.38.86.181', 'user_pass': ''},
    {'ip_port': '119.142.86.110', 'user_pass': ''},
    {'ip_port': '124.161.16.89', 'user_pass': ''},
    {'ip_port': '61.160.233.8', 'user_pass': ''},
    {'ip_port': '101.94.131.237', 'user_pass': ''},
    {'ip_port': '219.157.162.97', 'user_pass': ''},
    {'ip_port': '61.152.89.18', 'user_pass': ''},
    {'ip_port': '139.224.132.192', 'user_pass': ''}
]

class ZhipinSpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class ZhipinDownloaderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class ProxyMiddleware(object):
    def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']
            encoded_user_pass = base64.encodestring(proxy['user_pass'])
            request.headers['Proxy-Authorization'] = 'Basic ' + encoded_user_pass
        else:
            request.meta['proxy'] = "http://%s" % proxy['ip_port']


class RandomUserAgent(object):
    """
    随机获取settings.py中配置的USER_AGENTS设置'User-Agent'
    """

    def __init__(self, agents):
        self.agents = agents

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings.getlist('USER_AGENTS'))

    def process_request(self, request, spider):
        request.headers.setdefault('User-Agent', random.choice(self.agents))


class SeleniumMiddleware(object):
    def __init__(self):
        self.timeout = 120
        # 2.Chrome---------------------------------
        # 实例化参数对象
        options = webdriver.ChromeOptions()
        # 无界面
        options.add_argument('--headless')
        # 关闭浏览器弹窗
        prefs = {
            'profile.default_content_setting_values': {
                'notifications': 2
            }
         }
        options.add_experimental_option('prefs', prefs)
        # 打开浏览器
        self.browser = webdriver.Chrome(chrome_options=options)
        # 指定浏览器窗口大小
        self.browser.set_window_size(1366, 768)
        # 设置页面加载超时时间
        self.browser.set_page_load_timeout(self.timeout)
        self.wait = WebDriverWait(self.browser, self.timeout)
        print('SeleniumMiddleware __init__')

    def process_request(self, request, spider):

        # 获取页面
        self.browser.get(request.url)

        job_list = self.browser.find_elements_by_css_selector('.job-list > ul > li')
        # 切几条出来测试
        # job_list=job_list[0:3]
        # print(f"定位到元素组:{job_list}")
        # 循环遍历出每一条搜索结果的标题
        # 循环 的索引
        index=0
        for list_item in job_list:

            # execute_script 接收js 脚本 控制浏览器
            # scrollIntoView 滚动页面，直到元素进入视图。
            self.browser.execute_script(f"document.querySelectorAll('.job-list > ul > li')[{index}].scrollIntoView({{behavior: 'smooth', block: 'center', inline: 'center'}})")
            job_title = list_item.find_element_by_css_selector('div.job-title')
            # 模拟鼠标悬停
            ActionChains(self.browser).move_to_element(job_title).perform()
            try:
                # 显式等待 detail-bottom-text 出现
                detail_bottom_text=WebDriverWait(self.browser,10,1).until(lambda x:list_item.find_element_by_css_selector(".detail-bottom-text"))
                print(f"显式等待出现第{index+1}个元素成功:{detail_bottom_text}")
            except Exception as e:
                print(f"第{index+1}个元素显式等待超时:{e}")
                pass
            index+=1
            # 等待2秒
            time.sleep(2)


        # 等待5秒 
        time.sleep(2)
        # 返回页面的response
        print('process_request complete')
        return HtmlResponse(url=self.browser.current_url, body=self.browser.page_source,
                            encoding="utf-8", request=request)

    def scrollIntoView(self,request, spider):
        # scrollIntoView 滚动页面，直到元素进入视图。
        # 当请求的页面不是当前页面时
        if self.browser.current_url != request.url:
            time.sleep(5)
            # 请求的url开始为https://www.zhipin.cn/cn/时，调用滑动界面，每页20个，滑动4次
            if request.url.startswith("https://www.zhipin.cn/"):
                try:
                    for i in (0, 1, 2, 3):
                        # execute_script 接收js 脚本 控制浏览器
                        self.browser.execute_script(
                            # scrollIntoView 滚动页面，直到元素进入视图。
                            # "document.querySelector('cm-explain-bottom')[0].scrollIntoView(true)"
                            "document.getElementsByClassName('cm-explain-bottom')[0].scrollIntoView(true)")
                        time.sleep(4)
                # js 报错
                except JavascriptException as e:
                    pass
                except Exception as e:
                    pass
        else:
            pass
        pass

    def spider_closed(self):
        # 爬虫结束 关闭窗口
        self.browser.close()
        pass

    @classmethod
    def from_crawler(cls, crawler):
        # 设置爬虫结束的回调监听
        s = cls()
        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)
        return s